/***************************************************************************************************
  *Copyright(C),2010-2016,Sumscope
  *FileName	:  FieldCollector.cpp
  *Author	:  scofined.qi
  *Version	:  1.0
  *Date		:  2019/09/11
  *Desc		:  //?????????????????????????????
  *Relation :
  *Others	:  //???????????
  *Function :  //????????б???????????????????????????????
  *History	:  //??????????б???????????????????????????????????????
  ***************************************************************************************************/
#include "FieldCollector.h"
#include "StopCodec.h"
#include "common_util.h"
#include "TableCodec.h"
#include "BitViewWriter.h"
#include "portability.h"
//#include <stdlib.h>
#ifdef CCORE_LINUX
#include <malloc.h>
#elif CCORE_MAC
#include <sys/malloc.h>
#endif
namespace qb{
	namespace proto{

		FieldCollector::FieldCollector(kEncodeType type, int col, int row) :
			m_row(row), m_col(col), m_type(type), m_bitwidth(0), m_dictindex(-1),
			m_mincount(3), m_all_type(kExpectTypeNull)
		{
			m_values.reserve(row);
		}
		void FieldCollector::SetKey(const FieldKey& key){ 
			m_key = key;
		}


		int FieldCollector::GetBitWidth() const
		{
			if (m_type != kEncodeTypeFixedBit)
				return 0;
			return m_bitwidth;
		}

		bool FieldCollector::HasOpBitmap() const
		{
			return m_opbitmap.size() > 0;
		}
		void FieldCollector::SetOpBitmap(const OpBitmap& fieldOps)
		{
			m_opbitmap.clear();
			m_opbitmap = fieldOps;
		}
		void FieldCollector::AddValue(int row,const VarValue& value)
		{
			// ???????,???map??????????????key
			BASSERT(value.type != kExpectTypeNull);
			if (m_values.size() >= m_row || m_values.size()>=row+1)
				return;
			while (m_values.size() < row)
			{
				m_values.push_back(VarValue());
			}
			m_values.push_back(value);
			BASSERTS(m_values.size() == row+1, "row:%d count:%d", m_row, m_values.size());
			BASSERTS(m_values[row].EqualWith(value), "row:%d count:%d", m_row, m_values.size());
		}
		void FieldCollector::AddDefault(int row)
		{
			// ??????????е????????????,
			// ???????????в???????
			while (m_values.size()<row+1)
			{
				m_values.push_back(VarValue());
			}
			BASSERT(m_values.size() == row+1);
		}
		const VarValue& FieldCollector::GetValue(int index)
		{
			BASSERTS(index >= 0 && index < m_values.size(), "index:%d  count:%d", index, m_values.size());
			return m_values[index];
		}
		
		void FieldCollector::GetStat(FieldStat& stat) const
		{
			stat.name = m_key;
			stat.encode[m_type]++;
			if (m_opbitmap.size() > 0)
				stat.opcount[1]++;
			else
				stat.opcount[0]++;
			for (int i = 0; i < (int)m_values.size(); i++)
			{
				const VarValue& vv = m_values[i];
				stat.type[vv.type]++;
				switch (vv.type)
				{
				case kExpectTypeNull:stat.size.Add(0); break;
				case kExpectTypeString:stat.size.Add(vv.str.len); break;
				case kExpectTypeDouble:stat.size.Add(sizeof(double)); break;
				case kExpectTypeInt:
					if (vv.i < 256 && vv.i >= -127)
						stat.size.Add(sizeof(char));
					else if (vv.i<65536 && vv.i>-32768)
						stat.size.Add(sizeof(short));
					else if (vv.i<4294967296 && vv.i>-2147483647)
						stat.size.Add(sizeof(int));
					else
						stat.size.Add(sizeof(i64));
					break;
				}

			}
		}
		kExpectType FieldCollector::GetAllType()const
		{
			if (m_all_type == kExpectTypeNull)
				m_all_type = GetAllTypeImpl();
			return m_all_type;
		}
		bool FieldCollector::IsSameLength()const
		{
			// ?ж?????????????
			if (GetAllType() != kExpectTypeString || m_values.size()<=1)
				return false;
			const VarValue& first = m_values[0];
			if (first.type != kExpectTypeString)
				return false;
			for (int i = 1; i < (int)m_values.size(); i++)
			{
				const VarValue& vv = m_values[i];
				if (vv.type != kExpectTypeString ||
					vv.str.len != first.str.len)
					return false;
			}
			return true;


		}
		kExpectType FieldCollector::GetAllTypeImpl()const
		{
			// ??????????м?????,????????????
			// ???????????,??????????
			kExpectType type = kExpectTypeNull;
			if (m_values.size() > 0)
			{
				type = m_values[0].type;
				for (int i = 1; i < (int)m_values.size(); i++)
				{
					const VarValue& val = m_values[i];
					if (val.type == type)
						continue;
					if (val.type != kExpectTypeNull)
					{
						if (type == kExpectTypeNull)
							type = val.type;
						else
							return kExpectTypeNull;
					}
				}
				for (int i = 0; i < (int)m_values.size(); i++)
				{
					const VarValue& val = m_values[i];
					BASSERT(val.type == kExpectTypeNull || val.type == type);
				}
			}
			return type;
		}
		int FieldCollector::CalcEncodeSize(kEncodeType type, OpBitmap& useOpBitmap)const
		{
			// ????????OpBitmapFlag,?????????,?????????????
			//???nDefault>0, ????????useOpBitmap, sumSize??,?е????б???
			int sumSize = 0;
			typedef StackArray<int, 128> IntStackArray;
			IntStackArray calcSizes(m_values.size());
			//std::vector<int> calcSizes;
			//calcSizes.reserve(m_values.size());
			bool bHasDefault = false;
			useOpBitmap.clear();

			// ????????????????????????????????0??????????????useOpBitmap
			// ???????????????0???????????????????0??
			bool str2int = GetAllType() == kExpectTypeString && type == kEncodeTypeVarInt;

			for (int i = 0; i < (int)m_values.size(); i++)
			{
				const VarValue& val = m_values[i];
				int size = 1;//????????op,????????????1?????,??????????0???
				if (val.IsDefault())
				{
					// ????????????????????????,???""??"0"?????????0
					if (val.type == kExpectTypeString && type == kEncodeTypeVarInt)
					{
						return -1;
					}
					bHasDefault = true;
				}else
					size = StopCodec::CalcEncodeSize(type, val);
				if (str2int && !bHasDefault && val.type == kExpectTypeString && val.str.len == 1 && val.str.ptr[0] == '0')
					bHasDefault = true;
				if (size < 0)
					return -1;
				calcSizes[i]=size;
				sumSize += size;
			}
			// ?????????,????OpBitmapFlag?????????????
			// 1.???е?VarValue??type???????,???????????:??????????????
			// 2.??????????????
			// 3.??????,??????,?仯?С
			// 4.????,???????????????
			bool enableDelta = true;
			if (GetAllType() == kExpectTypeDouble)
				enableDelta = false;
			if (enableDelta && (bHasDefault || (m_values.size() > m_mincount && IsNumberType(GetAllType()))))
			{
				VarValue dftValue;
				const VarValue& first = m_values[0];
				int sumSizeUseOp = StopCodec::CalcEncodeSize(type, first);
				if (sumSizeUseOp > 0)
				{
					useOpBitmap.reserve(m_values.size());
					useOpBitmap[0] = first.EqualWith(dftValue) ? kOpTypeDefault : kOpTypeNew;
					for (int i = 1; i < (int)m_values.size(); i++)
					{
						const VarValue& val = m_values[i];
						unsigned int op = kOpTypeNew;
						if (val.EqualWith(dftValue))
							op = kOpTypeDefault;
						else if (val.EqualWith(m_values[i - 1]))
							op = kOpTypeCopy;
						else if (IsNumberType(val.type) && val.type == m_values[i - 1].type)
						{
							VarValue delta = val.Delta(m_values[i - 1], NULL);
							int size = calcSizes[i];//???????????
							int deltaSize = StopCodec::CalcEncodeSize(type, delta);
							if (deltaSize < size)
							{
								op = kOpTypeDelta;
								sumSizeUseOp += deltaSize;
							}
							else
							{
								op = kOpTypeNew;
								sumSizeUseOp += size;
							}
						}
						else
							sumSizeUseOp += StopCodec::CalcEncodeSize(type, val);
						useOpBitmap[i] = op;
					}
					BASSERTS(useOpBitmap.size() == m_values.size(), "useOpBitmap:%d m_values:%d", useOpBitmap.size(), m_values.size());
					// ????OpBitmapFlag?????????????λ??????,??????2bitλ
					int opbits = m_values.size()*OpBitWidth;
					int opsize = opbits % 7 == 0 ? (opbits / 7) : (opbits / 7 + 1);
					sumSizeUseOp += opsize;
					if (sumSizeUseOp < sumSize ||str2int)
					{
						return sumSizeUseOp;
					}
				}
			}
			else if (m_values.size()>1 && GetAllType() == kExpectTypeString && 
				(type == kEncodeTypeStrRaw || type == kEncodeTypeStrAlpNum || type == kEncodeTypeStrNum)
				&& m_values.size()>1)
			{
				VarValue dftValue;
				const VarValue& first = m_values[0];
				int sumSizeUseOp = StopCodec::CalcEncodeSize(type, first);
				if (sumSizeUseOp > 0)
				{
					useOpBitmap.reserve(m_values.size());
					useOpBitmap[0] = first.EqualWith(dftValue) ? kOpTypeDefault : kOpTypeNew;
					for (int i = 1; i < (int)m_values.size(); i++)
					{
						const VarValue& val = m_values[i];
						unsigned int op = kOpTypeNew;
						int deltaSize = 0;
						if (val.EqualWith(dftValue))
							op = kOpTypeDefault;
						else if (val.EqualWith(m_values[i - 1]))
							op = kOpTypeCopy;
						else
							op = (unsigned int)val.GetStringOp(m_values[i - 1], type, deltaSize);

						if (op == kOpTypeNew)
						{
							sumSizeUseOp += StopCodec::CalcEncodeSize(type, val);
						}
						else if (op == kOpTypeDelta)
						{
							sumSizeUseOp += deltaSize;
						}
						useOpBitmap[i] = op;
					}
					BASSERTS(useOpBitmap.size() == m_values.size(), "useOpBitmap:%d m_values:%d", useOpBitmap.size(), m_values.size());
					// ????OpBitmapFlag?????????????λ??????,??????2bitλ
					int opbits = m_values.size()*OpBitWidth;
					int opsize = opbits % 7 == 0 ? (opbits / 7) : (opbits / 7 + 1);
					sumSizeUseOp += opsize;
					if (sumSizeUseOp < sumSize)
					{
						return sumSizeUseOp;
					}
				}
			}
			useOpBitmap.clear();
			return sumSize;
		}
		kEncodeType FieldCollector::DeduceStringEncodeType(OpBitmap& useOpBitmap)const
		{
			const kEncodeType types[] = { kEncodeTypeVarInt, kEncodeTypeStrNum, kEncodeTypeStrAlpNum, kEncodeTypeStrRaw };
			int ntype = _countof(types);
			int predictSize = -1;
			OpBitmap opBitmap, useOpBitmapTemp;
			kEncodeType type = kEncodeTypeAuto;
			for (int i = 0; i < ntype; i++)
			{
				int size = CalcEncodeSize(types[i], useOpBitmapTemp);
				if (size<0)
					continue;
				if (predictSize < 0 || predictSize>size)
				{
					type = types[i];
					predictSize = size;
					opBitmap = useOpBitmapTemp;
					if (types[i] == kEncodeTypeStrNum)
						break;
				}
			}
			BASSERTS(type != kEncodeTypeAuto, "");
			useOpBitmap = opBitmap;
			return type;
		}
		kEncodeType FieldCollector::DeduceEncodeType(OpBitmap& useOpBitmap)const
		{
			/*
			OpBitmapFlag	???????????,??copy,delta,default???,??????????????,???????OpBitmapFlag
			DictIdFlag		?????????

			????:
				kEncodeTypeVarInt		
				kEncodeTypeStrNum
				kEncodeTypeStrAlpNum
				kEncodeTypeStrRaw

			???kEncodeTypeFixedBit

			??????????????kExpectType?????????????????????
			???????,????[A-Za-z0-9\.\-]????????kEncodeTypeStrAlpNum
					 ????[0-9\.\-]????????kEncodeTypeStrNum
					 ????[0-9]?????С??20,i64????????,????????kEncodeTypeVarInt
					 ???????????,??????????????λ,??????????
			??????:kEncodeTypeVarInt,kEncodeTypeVarUint
			??????:???kEncodeTypeDouble????
			*/
			int predictSize = -1;
			kEncodeType type = kEncodeTypeAuto;
			
			const kEncodeType typesAll[] = {
				kEncodeTypeStrRaw, kEncodeTypeStrAlpNum, kEncodeTypeStrNum,
				kEncodeTypeVarInt, kEncodeTypeVarUint,kEncodeTypeDouble
			};
			const kEncodeType typesStr[] = { kEncodeTypeStrRaw, kEncodeTypeStrAlpNum, kEncodeTypeStrNum, kEncodeTypeVarInt};
			const kEncodeType typesInt[] = { kEncodeTypeVarInt, kEncodeTypeVarUint };
			const kEncodeType typesDbl[] = { kEncodeTypeDouble };
			const kEncodeType typesDblStr[] = { kEncodeTypeDouble, kEncodeTypeStrNum/*????double??????????*/ };
			kExpectType all = GetAllType();
			const kEncodeType* types = typesAll;
			int ntype = _countof(typesAll);
			bool enableDoubleToStr = false;
			if (all == kExpectTypeString){
				types = typesStr; ntype = _countof(typesStr);
				//return DeduceStringEncodeType(useOpBitmap);
			}
			else if (all == kExpectTypeInt){
				types = typesInt; ntype = _countof(typesInt);
			}
			else if (all == kExpectTypeDouble){
				types = typesDbl; ntype = _countof(typesDbl);
				if (enableDoubleToStr)
				{
					types = typesDblStr; ntype = _countof(typesDblStr);
				}
			}
			OpBitmap opBitmap, useOpBitmapTemp;
			int i = 0;
			for (; i < ntype; i++)
			{
				int size = CalcEncodeSize(types[i], useOpBitmapTemp);
				if (size <= 0)
				{
					// ???????,??治??,?????????
					if(all == kExpectTypeString)
						break;
					continue;
				}
				if (predictSize < 0 || predictSize>size)
				{
					type = types[i];
					predictSize = size;
					opBitmap = useOpBitmapTemp;
				}
			}

			// ??????????????????,?????Ч,??????????????,????????????????
			if (type == kEncodeTypeAuto)
			{
				//BASSERTS(0, "????????????");
				return types[i];
			}
			useOpBitmap = opBitmap;
			return type;
		}
		int FieldCollector::Encode(StopCodec* encoder, TableCodec* allocator)
		{
			// ???????
			// ?????????? kEncodeTypeVarInt
			// double?????   kEncodeTypeStrNum

			if (m_type == kEncodeTypeAuto)
			{
				// ??????,????m_values?е??,????m_type
				m_type = DeduceEncodeType(m_opbitmap);
				BASSERT(m_opbitmap.size() == 0 || m_opbitmap.size() == m_values.size());
				BASSERT(m_type != kEncodeTypeAuto);
			}
			int pos = encoder->GetPos();
			bool hasOpBitmap = HasOpBitmap();
			bool bHasDict = HasDict();
			unsigned char type = (unsigned char)m_type;//??????7λ
			if (hasOpBitmap)
				type |= FieldFlagHasOpBitmap;
			if (bHasDict)
				type |= FieldFlagHasDictIndex;
			if ((m_type == kEncodeTypeVarInt && GetAllType() == kExpectTypeString) ||
				(m_type == kEncodeTypeStrNum && GetAllType() == kExpectTypeDouble)
				)
			{
				type |= FieldFlagHasConvert;
			}

			encoder->AddValue(kEncodeTypeVarInt, VarValue((unsigned int)type)); //base-128??????

			if (hasOpBitmap)
			{
				BASSERT(m_opbitmap.size() == m_values.size());
				//static_assert(OpBitWidth == 2, "OpBitWidth not equal to 2");
				int bits = m_opbitmap.size()*OpBitWidth;
				int opmax = 1 << OpBitWidth;
				int bytes = (bits%CHAR_BIT != 0) ? (bits / CHAR_BIT + 1) : (bits / CHAR_BIT);
				unsigned char* opBitmap = (unsigned char*)_alloca(bytes);
				memset(opBitmap, 0, bytes);
				BitWriter view(opBitmap, bytes, m_opbitmap.size(), 2, false);//???encoder?л???????λ
				for (int i = 0; i < m_opbitmap.size(); i++)
				{
					BASSERT(m_opbitmap[i] < opmax);
					view.set(i, m_opbitmap[i]);
				}
				encoder->AddValue(kEncodeTypeStrRaw, VarValue((const char*)opBitmap, bytes));
			}

			int bitWidth = GetBitWidth();
			if (bitWidth > 0)
				encoder->AddValue(kEncodeTypeVarInt, VarValue((unsigned int)bitWidth));

			// ???????÷??????m_type??????????
			// ????,???
			for (int i = 0; i < (int)m_values.size(); i++)
			{
				unsigned int op = hasOpBitmap ? m_opbitmap[i] : kOpTypeNew;
				switch (op)
				{
				case kOpTypeNew:encoder->AddValue(m_type, m_values[i]); break;
				case kOpTypeDefault:break;//????
				case kOpTypeCopy:break;//????
				case kOpTypeDelta:
					encoder->AddValue(m_type, m_values[i].Delta(m_values[i - 1],allocator));
					break;
				}
			}
			return encoder->GetPos() - pos;
		}
		bool FieldCollector::EqualValues(FieldCollector* fields)const
		{
			if (fields == NULL)
				return false;
			if (m_values.size() != fields->m_values.size())
				return false;
			for (int i = 0; i < (int)m_values.size();i++)
			{
				if (!m_values[i].EqualWith(fields->m_values[i]))
					return false;
			}
			return true;
		}

		void StatValue::Add(const StatValue& v)
		{
			imax = count <= 0 ? v.imax : (MAX(imax, v.imax));
			imin = count <= 0 ? v.imin : (MIN(imin, v.imin));
			count += v.count;
			sum += v.sum;
		}

		FieldKey::FieldKey(int col)
		{
			char buf[64] = { 0 };
			FMTBUF(buf, "%d", col);
			m_str = buf;
		}

		SeqConvertMap::SeqConvertMap()
			:m_lastvisit(-1)
		{

		}

		void SeqConvertMap::Init(std::vector<FieldKey>& vecKey)
		{
			BASSERT(vecKey.size() % 2 == 0);
			m_orgin.clear();
			m_target.clear();
			//???????????????
			for (int i = 0; i < vecKey.size(); i += 2)
			{
				m_orgin.push_back(vecKey[i]);
				m_target.push_back(vecKey[i + 1]);
			}
		}

		FieldKey* SeqConvertMap::ConvertFieldKey(const std::string& key)
		{
			int start = m_lastvisit + 1;
			if (m_lastvisit < 0 || m_lastvisit >= m_orgin.size())
				start = 0;
			for (int i = 0; i < m_orgin.size(); ++i)
			{
				int index = (i + start) % m_orgin.size();
				if (m_orgin[index] == key)
				{
					m_lastvisit = index;
					return &m_target[index];
				}
			}
			return NULL;
		}

	}//namespace base;
}//namespace ss;
